package com.ug.system.processor;

import com.ug.common.cache.redis.RedisCacheService;
import com.ug.common.constant.HotConstants;
import com.ug.common.utils.SpringContextHolder;
import com.ug.system.entity.AttributeValue;
import com.ug.system.entity.HtmlAttributeValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;
import us.codecraft.webmagic.selector.XpathSelector;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;

/**
 * @author: LY
 * @description: 52pojie热门信息获取
 */
public class PojieHotHtmlPageProcessor implements PageProcessor {

    private static final Logger logger = LoggerFactory.getLogger(PojieHotHtmlPageProcessor.class);
    private final RedisCacheService redisCacheService = SpringContextHolder.getBean(RedisCacheService.class);
    //  部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private final Site site = Site.me().setTimeOut(10000)
        .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
        .addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
        .addHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
        .addHeader("Accept-Encoding", "gzip, deflate, br")
        .addHeader("Referer", "https://www.52pojie.cn/")
        .addHeader("Connection", "keep-alive")
        .addHeader("Upgrade-Insecure-Requests", "1")
        .addHeader("Cache-Control", "max-age=0")  // 添加此行
        .addHeader("DNT", "1")  // 添加此行
        .addHeader("Sec-Fetch-Site", "same-origin")  // 添加此行
        .addHeader("Sec-Fetch-Mode", "navigate")  // 添加此行
        .addHeader("Sec-Fetch-User", "?1")  // 添加此行
        .addHeader("Sec-Fetch-Dest", "document")  // 添加此行
        .setRetryTimes(3)
        .setSleepTime(1000);

    @Override
    public void process(Page page) {
        logger.info("获取52pojie热门数据");
        String key = HotConstants.HOT_TYPE_POJIE.concat(HotConstants.HOT_TYPE_KEY);
        try {
            Selectable select = page.getHtml().$(".bm_c").select(
                    new XpathSelector("//th[@class='common']")
            );
            List<AttributeValue> attributeValues = new ArrayList<>();
            //  获取到的多个元素
            List<Selectable> nodes = select.nodes();
            //  循环获取对应标签数据
            for (Selectable node : nodes) {
                //  获取链接
                String link = "https://www.52pojie.cn/" + node.$(".xst", "href").get();
                //  获取标题
                String title = node.$(".xst", "text").get();
                //  获取热搜指数
                String value = "0";
                AttributeValue attributeValue = new AttributeValue(title, link, value);
                attributeValues.add(attributeValue);
            }
            redisCacheService.put(key, attributeValues, HotConstants.HOT_TIME_OUT, TimeUnit.HOURS);
            page.putField("attributeValues", attributeValues);
        }catch (Exception e){
            logger.error("获取52pojie热门数据异常：{}", e.getMessage());
            page.putField("attributeValues", new ArrayList<>());
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        HtmlAttributeValue pipeline = new HtmlAttributeValue();
        Spider.create(new PojieHotHtmlPageProcessor())
                //  从指定链接处开始抓
                .addUrl(HotConstants.POJIE_HOT_URL)
                //  结果保存
                .addPipeline(pipeline)
                //  开启5个线程抓取
                .thread(5)
                //  启动爬虫
                .run();
        System.out.println(pipeline.getCollected());
    }
}
